import java.io.*;
import java.net.*;
import java.util.*;
import java.security.Security;
import java.security.Provider;
import javax.net.ssl.*;
import local.util.MyPrint;
import java.text.DecimalFormat;

import local.util.MyStringFormat;
import local.util.MyPrint;

/**********************************************************************************
 * class HttpTableDataLoader
 * public setUrl (String)
 * public String getUrl ()
 * public void connectToUrl ()
 * public String getUrlReturnString ()
 * public void stringToVector (String)
 * public Vector getUrlReturnVector()
 * public void setStartString ( String )
 * public void setEndString ( String )
 * public Vector selectUsefulElements (Vector)
 * pbulic void setDelimHashtable(String delim_file_name)
 * oublic Hashtable getDelimHashtable()
 * public void checkVectorFormat( Vector input_v)
 * -- private void setRecordHeadVector(Vector input_v)
 * public Vector getRecordHeadVector ()
 * private void setRecordVectorHold( Vector input_v)
 * public Vector getRecordVectorHold()
 **********************************************************************************/
public class HttpTableDataLoader {    

    private static boolean debug = false;
    private static boolean save = false;
    private Vector recordHeadVector = new Vector ();
    private Vector recordVectorHold = new Vector ();
    private static Vector tickHolderV = new Vector (); // to hold tick name 

    private String myUrl = null;
    private String urlReturnString = new String();
    private Vector urlReturnVector= new Vector(10);

    private static String groupId;
    private static Hashtable myHashTable = new Hashtable();
    private static Hashtable tickHashTable = new Hashtable();
    private String startString= null;
    private String endString  = null;
    private Hashtable delimHashtable = new Hashtable ();

    public static void main (String[] args) {

	HttpTableDataLoader mydl = new HttpTableDataLoader ();
	String url_return_string = null;
	Vector url_return_vector = new Vector ();
	Vector table_head_v = new Vector ();
	Vector table_record_v_v = new Vector ();

	//mydl.setUrl("http://biz.yahoo.com/p/s_conamed.html");     //Sector
	
	if (args[0].equals("1"))
	mydl.setUrl("http://biz.yahoo.com/p/utilitconameu.html"); //1
	if (args[0].equals("2"))
	mydl.setUrl("http://biz.yahoo.com/p/transpconameu.html"); //2
	if (args[0].equals("3"))
	mydl.setUrl("http://biz.yahoo.com/p/technoconameu.html"); //3
	if (args[0].equals("4"))
	mydl.setUrl("http://biz.yahoo.com/p/servicconameu.html"); //4
	if (args[0].equals("5"))
	mydl.setUrl("http://biz.yahoo.com/p/healthconameu.html"); //5
	if (args[0].equals("6"))
	mydl.setUrl("http://biz.yahoo.com/p/financconameu.html"); //6
	if (args[0].equals("7"))
	mydl.setUrl("http://biz.yahoo.com/p/energyconameu.html"); //7
	if (args[0].equals("8"))
	mydl.setUrl("http://biz.yahoo.com/p/noncycconameu.html"); //8
	if (args[0].equals("9"))
	mydl.setUrl("http://biz.yahoo.com/p/cyclicconameu.html"); //9
	if (args[0].equals("10"))
	mydl.setUrl("http://biz.yahoo.com/p/congloconameu.html"); //10
	if (args[0].equals("11"))
	mydl.setUrl("http://biz.yahoo.com/p/capgdsconameu.html"); //11
	if (args[0].equals("12"))
	
	mydl.setUrl("http://biz.yahoo.com/p/basicmconameu.html"); //12

	mydl.connectToUrl();
	url_return_string = mydl.getUrlReturnString();
	url_return_string = url_return_string.replaceAll("<small>","");
	mydl.stringToVector (url_return_string);
	MyPrint.printVector(url_return_vector,1);
	url_return_vector = mydl.getUrlReturnVector();
	mydl.setStartString("Industries");
	mydl.setEndString("Copyright");
	url_return_vector = mydl.selectUsefulElements (url_return_vector, false);
	//MyPrint.printVector(url_return_vector,1);
	MyPrint.printVector(url_return_vector,10,1);

	//mydl.checkVectorFormat(url_return_vector);
	//mydl.setRecordHeadVector(url_return_vector);
	//table_head_v = mydl.getRecordHeadVector ();
	//mydl.setRecordVectorHold(url_return_vector);
	//table_record_v_v = mydl.getRecordVectorHold();
    }

    public void setStartString ( String start_str ) {
	startString = start_str;
    }

    public void setEndString ( String end_str ) {
	endString = end_str;
    }

    public Vector sumColumnValue (Vector input_v_v, int index, String data_type, 
				  String start_with, int index_check) {
	Object return_type = null;
	Vector temp_v = new Vector();
	String temp_str = null;
	String check_str = null;
	double temp_double = 0.0;
	double total_double = 0.0;
	int    counter_double = 0;
	long   temp_long = 0;
	long   total_long = 0;
	int    counter_long = 0;
	int    temp_int = 0;
	int    total_int = 0;
	int    counter_int = 0;

	String return_value = null;
	Vector sum_vector = new Vector ();

	Enumeration enum = input_v_v.elements();
	while (enum.hasMoreElements()) {

	    //MyPrint.printVector(input_v_v, 1);
	    temp_v    = (Vector)enum.nextElement();
	    //MyPrint.printVector(temp_v, 1);
	    temp_str  = (String)temp_v.elementAt(index);
	    check_str = (String)temp_v.elementAt(index_check);

	    temp_str = temp_str.replaceAll("%","");
	    temp_str = temp_str.replaceAll(",","");
	    temp_str = temp_str.replaceAll("UNCH","0");
	    temp_str = temp_str.replaceAll("n/a","0");
	    temp_str = temp_str.replaceAll("N/A","0");

	    //System.out.println("temp_str="+temp_str);
	    //System.out.println("check_str="+check_str);

	    if (check_str.startsWith(start_with) || start_with.equals("any")) {
		if (data_type.equals("double")) {
		    temp_double =  Double.valueOf(temp_str).doubleValue();
		    total_double += temp_double;
		    counter_double++;
		}
		else if (data_type.equals("long")) {
		    temp_long =  Long.valueOf(temp_str).longValue();
		    total_long += temp_long;
		    counter_long++;  
		}
		else if (data_type.equals("int")) {
		    temp_int =  Integer.valueOf(temp_str).intValue();
		    total_int += temp_int;
		    counter_int++;  
		}
		else {
		    System.out.println ("sumColumnValue:No such data type");
		    System.exit(0);
		}
	    }
	    else {
		//System.out.println("check_str not nuch="+check_str);
	    }
	}


	if (data_type.equals("double")) {
	    sum_vector.addElement(Double.toString ( total_double  ));
	    sum_vector.addElement(Integer.toString( counter_double));
	}
	else if (data_type.equals("long")) {
	    sum_vector.addElement(Long.toString   (  total_long   ));
	    sum_vector.addElement(Integer.toString( counter_long  ));
	}
	else if (data_type.equals("int")) {
	    sum_vector.addElement(Integer.toString(   total_int   ));
	    sum_vector.addElement(Integer.toString(  counter_int  ));
	}

	return sum_vector;
    }

    public void setUrl (String url) {

	myUrl = url;
    }

    public String getUrl () {

	return myUrl;
    }

    public void connectToUrl () {

	String url_return_string = "";

	URL url ;


	try {            
	    url = new URL (myUrl);

	    try {                

		StringBuffer buff1= new StringBuffer("");

		if (myUrl.startsWith("https:")) {
		    HttpsURLConnection httpsConnection = null;
		    httpsConnection = (HttpsURLConnection)url.openConnection();                
		    httpsConnection.setDoOutput(false);                
		    httpsConnection.connect();                

		    if (debug) {
			System.out.println("Connection Code and Message:" + 
					   httpsConnection.getResponseCode() + 
					   " - " + httpsConnection.getResponseMessage()+"\n");
		    }

		    BufferedReader in = new BufferedReader(new InputStreamReader
			(httpsConnection.getInputStream()));


		    url_return_string = in.readLine();
		    
		    while (url_return_string != null) {
			buff1.append(url_return_string);
			url_return_string = in.readLine();
		    }
		    
		    in.close();            
		}
		else if (myUrl.startsWith("http:")) {   

		    HttpURLConnection httpConnection = null;
		    httpConnection = (HttpURLConnection)url.openConnection();                
		    httpConnection.setDoOutput(false);                
		    httpConnection.connect();                

		    if (debug) {
			System.out.println("Connection Code and Message:" + 
					   httpConnection.getResponseCode() + 
					   " - " + httpConnection.getResponseMessage()+"\n");
		    }

		    BufferedReader in = new BufferedReader(new InputStreamReader
			(httpConnection.getInputStream()));

		    url_return_string = in.readLine();
		    
		    while (url_return_string != null) {
			buff1.append(url_return_string);
			url_return_string = in.readLine();
		    }

		    in.close();            
		}

		url_return_string = buff1.toString();
		url_return_string = url_return_string.replaceAll("\n","");
		url_return_string = url_return_string.replaceAll("\t","");
		url_return_string = url_return_string.replaceAll("&nbsp;","");
		url_return_string = url_return_string.replaceAll("<NOBR>","");
		url_return_string = url_return_string.replaceAll("<small>","");
		url_return_string = url_return_string.replaceAll("&amp;","");
		url_return_string = url_return_string.replaceAll("<br>","");
		url_return_string = url_return_string.replaceAll(">",">\n");
		url_return_string = url_return_string.replaceAll("</","\n</");
		
		urlReturnString = url_return_string;

	    } catch (IOException ioe) {                
		System.out.println(ioe.getMessage());  
		System.exit(1);            
	    }        
	} catch (MalformedURLException mue) {            
	    System.out.println(mue.getMessage());            
	    System.exit(1);        
	} catch (Exception e) {            
	    e.printStackTrace();            
	    System.exit(1);        
	}    
    }

    public String getUrlReturnString () {
	return urlReturnString;
    }

    public void stringToVector (String input) {

	Vector url_return_vector = new Vector ();
	String element = null;
	StringTokenizer tokens = new StringTokenizer(input, "\n");
	while (tokens.hasMoreTokens()) {
	    element = tokens.nextToken();
	    if (element != null) {
		url_return_vector.addElement(element);
	    }
	}
	setUrlReturnVector(url_return_vector);
    }

    public Vector getUrlReturnVector() {
	return urlReturnVector;
    }

    public void setUrlReturnVector(Vector v_in) {
	urlReturnVector = v_in;
    }

    public Vector selectUsefulElements (Vector v_in, boolean include) {

	Vector v_out = new Vector (10);
	String text = null;
	Enumeration penum = v_in.elements();
       
	while ( penum.hasMoreElements()) {

	    text = (String)penum.nextElement();
	    if (text.startsWith(startString)) {
		if (include) {
		    v_out.addElement(text);
		}
		break;
	    }
	}
	
	while ( penum.hasMoreElements() ) {
	    text = (String)penum.nextElement();
	    if (text.startsWith(endString)) {
		break;
	    }
	   	    
	    if ( ! (text.startsWith("<") )) {
		v_out.addElement(text);
	    }
	}
	
	return v_out;
    }

    public Vector selectUsefulElements (Vector v_in) {

	Vector v_out = new Vector (10);
	String text = null;
	Enumeration penum = v_in.elements();
       
	while ( penum.hasMoreElements()) {

	    text = (String)penum.nextElement();
	    if (text.startsWith(startString)) {
		v_out.addElement(text);
		break;
	    }
	}
	
	while ( penum.hasMoreElements() ) {
	    text = (String)penum.nextElement();
	    if (text.startsWith(endString)) {
		break;
	    }
	   	    
	    if ( ! (text.startsWith("<") )) {
		v_out.addElement(text);
	    }
	}
	
	return v_out;
    }

    public void setDelimHashtable(String delim_file_name) {
  
	//String tick_file_name = "./Data/AllTick.txt";
	String file_name = delim_file_name;
 
	File delim_f;
	Hashtable tick_hashtab = new Hashtable();
	TickFileLoader my_tick_file = new TickFileLoader();

	if ((delim_f = new File (file_name)).isFile()) {
	    my_tick_file.setFileName(delim_f);
	    my_tick_file.loadFile();
	    tick_hashtab = my_tick_file.getHashTable();
	}
	else {
	    System.out.println("Can not find delimit file:"+delim_file_name);
	}

	delimHashtable = tick_hashtab;
    }

    public Hashtable getDelimHashtable () {
	return delimHashtable;
    }

    public void checkVectorFormat( Vector input_v) {
   
	int length[] = {0, 0, 0};
	int number_column = 0;
	Object temp_o, temp_o1;
	String tick, tick1;

	Hashtable tick_hashtab = new Hashtable();
	tick_hashtab = getDelimHashtable ();

	Vector record_head_vector = new Vector ();

	Enumeration enum = input_v.elements();
	int m = 0;   
	while ( enum.hasMoreElements()) {
	 
	    temp_o = enum.nextElement();
	    
	    tick = (String)temp_o;
	  
	    if (tick_hashtab.containsKey(tick)){

		length[m]=input_v.indexOf(tick);
		m++;
		if (m > 2) {
		    if((length[1]-length[0]) ==  (length[2]-length[1])) {
			number_column = length[1]-length[0];
			//System.out.println("number of column is:"+number_column);
			break;
		    }
		    else {
			m = 0;
		    }
		}
		else {
		    continue;
		}
	    }
	    else {
		if (length[0]==0){
		    record_head_vector.add(tick);
		}

	    }
	}

	if (record_head_vector.size() != number_column) {
	    System.out.println("Number of table column error!");
	    System.out.println("Record Head Size="+record_head_vector.size());
	    System.out.println("Number of Column=="+number_column);
	    System.out.println(record_head_vector);
	    System.exit(0);
	}
	else {
	    setRecordHeadVector(record_head_vector);
	}
    }

    public void setRecordVectorHold( Vector input_v) {

	int index = 0;
	int next_index = 0;
	int number_column = 0;
	Object temp_o, temp_o1;
	String tick, tick1;

	Vector record_v_v = new Vector ();
	Hashtable tick_hashtab = new Hashtable ();
	tick_hashtab = getDelimHashtable ();

	Enumeration enum1 = input_v.elements();

	while ( enum1.hasMoreElements()) {
	    temp_o1 = enum1.nextElement();
	    
	    tick1 = (String)temp_o1;

	    if (tick_hashtab.containsKey(tick1)) {
		index = input_v.indexOf(tick1);
		next_index = index + number_column;
 
		if (next_index < input_v.size()) {
		    if (tick_hashtab.containsKey((String)input_v.elementAt(next_index))){
			//System.out.println("1111   "+tick1);
			Vector record_v   = new Vector ();
			record_v.addElement(tick1);
			for (int j =1; j <number_column; j++) {
			    record_v.addElement((String)enum1.nextElement());
			}

			record_v_v.addElement(record_v);
		    }
		}
		else if (next_index == input_v.size()){
		    //System.out.println("2222   "+tick1);
		    Vector record_v   = new Vector ();
		    record_v.addElement(tick1);

		    for (int j =1; j <number_column; j++) {
			record_v.addElement((String)enum1.nextElement());
		    }

		    record_v_v.addElement(record_v);
		}
	    }
	    else {
		continue;
	    }
	}

	recordVectorHold = record_v_v;
    }

    private void setRecordHeadVector(Vector input_v) {
	recordHeadVector = input_v;
    }

    public Vector getRecordHeadVector () {
	return recordHeadVector;
    }

    public Vector getRecordVectorHold () {
	return recordVectorHold;
    }

    private void insertIntoDataBase( Vector inputV) {

	StringBuffer queryBf = new StringBuffer("INSERT INTO sector_record VALUES (");


	int m =0; int n = 0; String temp;  double change=0.00;
	Enumeration penum = inputV.elements();

	while ( penum.hasMoreElements()) {
	    m++; 
	    temp = (String)penum.nextElement();
	    //System.out.print(temp+"\t");

	    if (m%8==0 && m != 0)  System.out.println();

	    if (m >8) {

	       temp = temp.replaceAll("%","");
	       temp = temp.replaceAll(",","");
	       temp = temp.replaceAll("UNCH","0");
	       temp = temp.replaceAll("n/a","0");

	       queryBf.append("'"+temp+"',");

	       if (m%8==0) {
		   queryBf.append("'"+groupId.trim()+"','"+myHashTable.get(groupId)+"'"+");");
		   //System.out.println(queryBf.toString());
		   DataBaseInsert myInsert = new DataBaseInsert();	
		   myInsert.setQueryString(queryBf.toString());

		   if(myInsert.insertData() == 1) {
		       queryBf = new StringBuffer("INSERT INTO sector_record VALUES (");
		   }
		   else {
		       do {
			   myInsert.setQueryString(queryBf.toString());
		       } while (myInsert.insertData()!=1);

		       queryBf = new StringBuffer("INSERT INTO sector_record VALUES (");
		   }
	       }
	    }
	}
    }    

    private void insertIntoDataBase( String queryStr) {
	int tryTime =0;
	StringBuffer queryBf = new StringBuffer("INSERT INTO sector_record VALUES (");

	queryBf.append(queryStr+");");

	DataBaseInsert myInsert = new DataBaseInsert();	
	
	myInsert.setQueryString(queryBf.toString());
	myInsert.insertData();

    }         
}








